#!/bin/bash

INFOMAIL="yajin@corp.ebay.com"
ALERTMAIL="yajin@corp.ebay.com"

LOG="/ebay/bind-logs/bind-misc.log"
ERROR="refresh: retry limit for master"
GREP="/bin/grep"
TIMES="/var/named/tmp/SOA_QUERY_HANGING"

export PATH=/opt/bind/bin:/opt/bind/sbin:/bin:/usr/bin:/sbin:/usr/sbin

typeset -i no times=0

tail -100 $LOG | $GREP -q  "$ERROR"
if [ $? -eq 0 ]; then 
	no=$(rndc status | $GREP "soa queries in progress:" | awk -F":" '{print $2}')
	if [ $no -ge 50 ]; then 
		if [ ! -f $TIMES ]; then 
			echo 1 > $TIMES
			echo "reloading named configuration..."
			rndc reconfig
			if [ $? -ne 0 ]; then 
				echo "reloading named configuration failed" >&2
				echo "Job $0 at $(date) on $(hostname)." | mailx -s "[unixmon]: reloading named configuration failed" $ALERTMAIL
			fi
		else
			times=$(cat $TIMES)+1
			if [ $times -ge 3 ]; then 
				echo "restarting named..."
				service named restart 
				if [ $? -ne 0 ]; then 
					echo "restarting named failed" >&2
					echo "Job $0 at $(date) on $(hostname)." | mailx -s "[unixmon]: restarting named failed" $ALERTMAIL
				else
					echo "Job $0 at $(date) on $(hostname)." | mailx -s "[unixmon_info} restarting named succeeded" $INFOMAIL
				fi
				rm -f $TIMES
			else
				echo "reloading named configuration..."
				rndc reconfig
				if [ $? -ne 0 ]; then 
					echo "reloading named configuration failed" >&2
					echo "Job $0 at $(date) on $(hostname)." | mailx -s "[unixmon]: reloading named configuration failed" $ALERTMAIL
				fi
				echo $times > $TIMES
			fi
		fi
	fi
else 
	if [ -f $TIMES ]; then rm $TIMES; fi
fi
